package com.crawler;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;


public class Crawler {
	private String url;
	private int maxRetry=3;
	private Document doc;
	public Crawler(String url){
		this.url=url;
	}
	public Boolean loadPage() {
		Boolean success=false;
		Boolean retry=true;
		int retryNum=0;
		while(retry){
			String html=HttpClientHelper.getHtml(url);
			if(html==null){
				retryNum++;
				if(retryNum>=maxRetry){
					retry=false;
				}else {
					try {
						Thread.sleep(3000);
					} catch (InterruptedException e1) {
						// TODO Auto-generated catch block
						e1.printStackTrace();
					}
				}
			}else {
				doc = Jsoup.parse(html);
				retry=false;
				success=true;
			}
		}
		return success;
	}
	public String getUrl() {
		return url;
	}
	public void setUrl(String url) {
		this.url = url;
	}

	public int getMaxRetry() {
		return maxRetry;
	}
	public void setMaxRetry(int maxRetry) {
		this.maxRetry = maxRetry;
	}
	public Document getDoc() {
		if(doc==null){
			loadPage();
		}
		return doc;
	}
	public void setDoc(Document doc) {
		this.doc = doc;
	}
	
	
	
	
}
